# -*- coding: utf-8 -*-
import scrapy
from scrapy import Selector

from novel.items import NovelItem


class NovelSpider(scrapy.Spider):
    name = 'novel'
    allowed_domains = ['xbiquge.la']
    start_urls = ['http://www.xbiquge.la/xiaoshuodaquan/']

    def parse(self, response):
        # print('进入')
        selector = Selector(response=response)
        link_urls = selector.xpath('/html/body/div[@id="wrapper"]/div[@id="main"]/div[@class="novellist"]')
        for link_url in link_urls:
            # catory_text = link_url.xpath('.//h2/text()').extract()
            catory_links = link_url.xpath('.//ul/li/a')
            for novel_link in catory_links:
                # print(novel_link)
                novel_text = novel_link.xpath('.//text()').extract_first()
                novel_url = novel_link.xpath('.//@href').extract_first()
                # print(novel_text + "    " + novel_url)
                yield scrapy.Request(novel_url, meta={'novel_name': novel_text}, callback=self.parse_novel_detail)
            # print(novel_link.xpath('/@href').extract())

    def parse_novel_detail(self, response):
        novel_name = response.meta['novel_name']
        selector = Selector(response)
        chapter_links = selector.xpath('//div[@class="box_con"]/div[@id="list"]/dl/dd/a')
        for chapter_link in chapter_links:
            chapter_title = chapter_link.xpath('.//text()').extract_first()
            chapter_url = chapter_link.xpath('.//@href').extract_first()
            # print(chapter_title, chapter_url)
            url = 'http://www.xbiquge.la' + chapter_url
            # print(url)
            yield scrapy.Request(url=url,
                                 meta={'novel_name': novel_name, 'chapter_title': chapter_title},
                                 callback=self.parse_novel_detail_chapter)

    def parse_novel_detail_chapter(self, response):
        chapter_title = response.meta['chapter_title']
        novel_name = response.meta['novel_name']

        selector = Selector(response=response)
        content = selector.xpath('//div[@class="content_read"]/div[@class="box_con"]/div[@id="content"]/text()').extract()
        content_total = ''
        for item_content in content:
            content_total += '\n'+item_content

        print(novel_name, chapter_title, content_total)
        novel_item = NovelItem()
        novel_item['name'] = novel_name
        novel_item['chapter_title'] = chapter_title
        novel_item['chapter_content'] = content_total
        return novel_item
